#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/1/31 10:50
# @Author  : 王凯
# @File    : project_setting.py
# @Project : scrapy_spider
# -*- coding: utf-8 -*-
"""爬虫配置文件"""
import os
import sys
from importlib import import_module
from pathlib import Path

BASE_DIR = Path(__file__).parent.as_posix()

config_modules = [
    "components.settings.private.wfq_dev_redis_settings",
    "components.settings.private.wfq_source_mysql_settings",
]
for module in config_modules:
    module = import_module(module)
    for config in dir(module):
        if config == config.upper():
            locals()[config] = getattr(module, config)
# MYSQL
MYSQL_IP = os.getenv("MYSQL_IP")
MYSQL_PORT = int(os.getenv("MYSQL_PORT", 3306))
MYSQL_DB = os.getenv("MYSQL_DB")
MYSQL_USER_NAME = os.getenv("MYSQL_USER_NAME")
MYSQL_USER_PASS = os.getenv("MYSQL_USER_PASS")

# REDIS
# ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2
REDISDB_IP_PORTS = os.getenv("REDISDB_IP_PORTS")
REDISDB_USER_PASS = os.getenv("REDISDB_USER_PASS")
REDISDB_DB = int(os.getenv("REDISDB_DB", 0))
# 适用于redis哨兵模式
REDISDB_SERVICE_NAME = os.getenv("REDISDB_SERVICE_NAME")

ITEM_FILTER_ENABLE = False  # item 去重
# item在内存队列中最大缓存数量
ITEM_MAX_CACHED_COUNT = 5000
# item每批入库的最大数量
ITEM_UPLOAD_BATCH_MAX_SIZE = 1000
# item入库时间间隔
ITEM_UPLOAD_INTERVAL = 1
# 内存任务队列最大缓存的任务数，默认不限制；仅对AirSpider有效。
TASK_MAX_CACHED_SIZE = 0

# 代理的redis
PROXY_REDISDB_IP_PORTS = "server-forward.local.wfq2020.com:56383"
PROXY_REDISDB_USER_PASS = "e63ceeb55115dbf72f4dcfdf44352604"
PROXY_REDISDB_DB = "0"

os.environ["PROXY_REDISDB_IP_PORTS"] = PROXY_REDISDB_IP_PORTS
os.environ["PROXY_REDISDB_USER_PASS"] = PROXY_REDISDB_USER_PASS
os.environ["PROXY_REDISDB_DB"] = PROXY_REDISDB_DB

PINZHAN_IP_REDIS_URL = f"redis://:{PROXY_REDISDB_USER_PASS}@{PROXY_REDISDB_IP_PORTS}/{PROXY_REDISDB_DB}"
SQUID_URL = "https://tmp.wfq2020.com/proxies/"
JS_ENV_SERVER_URL = "https://tmp.wfq2020.com/js-env-server" if sys.platform == "win32" else "http://js-env-server-dev:3000"

LOG_LEVEL = "INFO"
os.environ.setdefault("LOG_LEVEL", LOG_LEVEL)
os.environ.setdefault("LOGURU_LEVEL", LOG_LEVEL)
